*-------------------------------------------------------------------------------	
* Prepare datasets for regression
cap program drop est_ready_PW
program define est_ready_PW
args y1 y2
	
	use "$Data/est_ready.dta", clear
	
	*cityid_RPS == 58
	*Takamatsu = 58 population = 420,748
	
	* prepare log prices
	qui gen lp_prior=log(ptilde) if year == `y1'
	qui gen blp_prior_temp = log(ptilde) if year == `y1' & cityid_RPS == 58
	bys RPS year: egen blp_prior = total(blp_prior_temp)
	qui gen tmp_prior = blp_prior_temp - blp_prior
	qui sum tmp_prior
	assert `r(mean)' == 0
	
	qui gen lp_post=log(ptilde) if year == `y2' 
	qui gen blp_post_temp = log(ptilde) if year == `y2' & cityid_RPS == 58
	bys RPS year: egen blp_post = total(blp_post_temp)
	qui gen tmp_post = blp_post_temp - blp_post
	qui sum tmp_post
	assert `r(mean)' == 0
	
	qui gen avglp1 = lp_prior - blp_prior
	qui replace avglp1=. if missing(lp_prior)
	qui gen avglp2 = lp_post - blp_post 
	qui replace avglp2=. if missing(lp_post)
	
	qui label var avglp1 "quality adjusted log price in `y1' rel to Takamatsu"
	qui label var avglp2 "quality adjusted log price in `y2' rel to Takamatsu"
	
	
	* save variable labels
	foreach v of var avglp1 avglp2  te  TN_DW x_NS* xcat_NS* xdd_NS* xdds_NS* cityid_RPS RPS{
		local l`v' : variable label `v'
        if `"`l`v''"' == "" {
 		local l`v' "`v'"
		}
	}
	
	* collapse to have unique values by RPS and cityid_RPS
	collapse (mean) avglp1 avglp2 te TN_DW x_NS* xcat_NS* xdd_NS* xdds_NS*, by(RPS cityid_RPS)
	
	*label variabless
	foreach v of var * {
		label var `v' "`l`v''"
	}
  
	gen year1 = `y1'
	gen year2 = `y2'
	local x = `y2'-`y1'
	
	*drop Takamatsu
	*drop if cityid_RPS == 58

	* generate price change
	qui gen delta_avgp = avglp2 - avglp1
	drop if missing(delta_avgp)

	* discard incomplete observations
	qui keep if !missing(avglp1) & !missing(avglp2)
			
	* e-commerce intensity times price in the previous period
	*qui gen xlp1_rakuten = x_rakuten*avglp1 /* Rakuten*/
	*label var xlp1_rakuten "\$ x(Rakuten) \times \Delta p_{ict-`x'}$"
	foreach year in 2004 2009 2014 {
		qui gen xlp1_NS`year' = x_NS`year'*avglp1 
		label var xlp1_NS`year' "\$ x(NS`year') \times \Delta p_{ict-`x'}$"
	}
	
	* category intensity times price in the previous period
	foreach year in 1999 2004 2009 2014 {
		qui gen xcatlp1_NS`year' = xcat_NS`year'*avglp1 
		label var xcatlp1_NS`year' "\$ xcat(NS`year') \times \Delta p_{ict-`x'}$"
	}
	
	* merge METI numbers
	merge m:1 year2 using "$Data/METI_numbers", keep(1 3) nogen
	
	* label variables
	label var delta_avgp  "\$\Delta p_{ict}-\Delta p_{iTt}$"
	label var avglp1	"\$ p_{ic`y1'} - p_{iT`y1'} $"
	local var te 		"\$\tilde{e}$"
	
	
	* save datasets
	qui cap mkdir "$Data/pricereg"
	cd "$Data/pricereg"
	save "`y1'_`y2'_PWreg.dta", replace

end 

*==============================================================================
* One period regression specification
cap program drop pricereg_PW
program define pricereg_PW
args y1 y2 def fe goods iv dummy

	if "`fe'" == "city_product"{
		local fixed_effect = "citydum_t product_t"
	}
	if "`fe'" == "product"{
		local fixed_effect = "product_t"
	}
	
	use "$Data/pricereg//`y1'_`y2'_PWreg.dta", clear
	drop if TN_DW == 1 & "`goods'" == "tr"

	egen citydum_t = group(cityid_RPS year2)
	egen product_t = group(RPS year2)
	
	egen city_gr=group(cityid_RPS)
	egen product_gr = group(RPS)
	
	* limit to products and cities that exist in all relevant years
	foreach year in `y2' {
		cap qui gen b_city_`year' = 1 if (year2 == `year')& avglp1 ~= .
		sort cityid_RPS b_city_`year'
		cap qui by cityid_RPS: replace b_city_`year' = b_city_`year'[_n-1] if missing(b_city_`year') & _n~= 1
		
		cap qui gen b_product_`year' = 1 if (year2 == `year') & avglp1 ~= .
		sort RPS b_product_`year'
		cap qui by RPS: replace b_product_`year' = b_product_`year'[_n-1] if missing(b_product_`year') & _n~= 1
	}
	
	ds b_product* b_city*
	foreach var in `r(varlist)'{
		qui keep if `var' == 1
	}
	
	* generate dummy for period 2 interacte with xp
	if "`dummy'"=="d_1997"{
		qui gen dummy = 1 if year2>= 1997
		qui replace dummy = 0 if dummy == .
	}
	if "`dummy'"=="METI_size"{
		qui gen dummy = METI_log_size
		qui replace dummy = 0 if dummy == .
	}
	if "`dummy'"=="METI_share"{
		qui gen dummy = METI_share
		qui replace dummy = 0 if dummy == .
	}

	
	if "`def'" ==  "x_rakuten" {
		gen xlp=xlp1_rakuten
		gen d_xlp1 = xlp*dummy
		local x_year "2010"
	}
	if "`def'" == "x_NS2004" {
		gen xlp=xlp1_NS2004
		gen d_xlp1 = xlp*dummy
		local x_year "2004"
	}
	if "`def'" == "x_NS2009" {
		gen xlp=xlp1_NS2009
		gen d_xlp1 = xlp*dummy
		local x_year "2009"
	}
	if "`def'" == "x_NS2014" {
		gen xlp=xlp1_NS2014
		gen d_xlp1 = xlp*dummy
		local x_year "2014"
	}
	
	gen xcatlp = xcatlp1_NS1999
	gen d_xcatlp1 = dummy*xcatlp1_NS1999
	gen d_p = dummy*avglp1
	
	
	if `y2' >= 1997 & "`iv'" == "ols"{ 
		eststo: reghdfe delta_avgp avglp1 d_xlp1, absorb(`fixed_effect', save) vce(cluster cityid_RPS RPS)
		estadd local iv "OLS"
		estadd scalar r_squared `e(r2)'
	}
	if `y2' < 1997 &  "`iv'" == "ols"{ 
		eststo: reghdfe delta_avgp avglp1 xlp, absorb(`fixed_effect', save) vce(cluster cityid_RPS RPS)
		estadd local iv "OLS"
		estadd scalar r_squared `e(r2)'
	}
	if `y2' >= 1997 &  "`iv'" == "2sls"{ 			
		eststo: ivreghdfe delta_avgp avglp1 (d_xlp1=d_xcatlp1), absorb(`fixed_effect') cluster(cityid_RPS RPS) savefirst first saverf 
		estadd scalar fs = e(widstat)
		estadd local iv "IV"		
	}
	if `y2' < 1997 &  "`iv'" == "2sls"{ 				
		eststo: ivreghdfe  delta_avgp avglp1 (xlp=xcatlp), absorb(`fixed_effect') cluster(cityid_RPS RPS) savefirst first saverf
		estadd scalar fs = e(widstat)
		estadd local iv "IV"	
	}
	if `y2' >= 1997 &  "`iv'" == "2sls_2iv"{ 			
		eststo: ivreghdfe delta_avgp avglp1 (d_xlp1=d_xcatlp1 d_xraklp1), absorb(`fixed_effect') cluster(cityid_RPS RPS) savefirst first saverf
		estadd scalar fs = e(widstat)
		estadd local iv "IV (Cat, Rak)"
		estadd scalar jstat = e(j)
		estadd scalar jpval = e(jp)
	}
	if `y2' < 1997 &  "`iv'" == "2sls_2iv"{ 				
		eststo: ivreghdfe  delta_avgp avglp1 (xlp=xcatlp xraklp), absorb(`fixed_effect') cluster(cityid_RPS RPS) savefirst first saverf
		estadd scalar fs = e(widstat)
		estadd local iv "IV (Cat, Rak)"	
		estadd scalar jstat = e(j)
		estadd scalar jpval = e(jp)
	}
	estadd local pr "\{`y2'\}"
	estadd local "x_t" `x_year'
	estadd local k `=`y2'-`y1''
	
end

*==============================================================================
* Diff-in-Diff regression  specification
cap program drop reg_dind_PW
program define reg_dind_PW
args y1 y2 y3 y4 def fe goods iv dummy

*reg_dind 1991 1996 1996 2001 `def' "`fe'" tr `est' "`dummy'"

	if "`fe'" == "city_product"{
		local fixed_effect = "citydum_t product_t"
	}
	if "`fe'" == "product"{
		local fixed_effect = "product_t"
	}

	cd "$Data/pricereg"
	use "`y1'_`y2'_PWreg.dta", clear
	append using "`y3'_`y4'_PWreg.dta"

	* keep tradables
	*drop if TN_DW == 1 & "`goods'" == "tr"

	* generate variable for fixed effects that vary for each year
	egen citydum_t = group(cityid_RPS year2)
	egen product_t = group(RPS year2)
			
	* find the product, city, and product-city pairs that exist in each year
	forvalues i = 2(2)4 {
		cap qui gen b_city_`y`i'' = 1 if (year2 == `y`i'')& avglp1 ~= .
		sort cityid_RPS b_city_`y`i''
		cap qui by cityid_RPS: replace b_city_`y`i'' = b_city_`y`i''[_n-1] if missing(b_city_`y`i'') & _n~= 1
		
		cap qui gen b_product_`y`i'' = 1 if (year2 == `y`i'') & avglp1 ~= .
		sort RPS b_product_`y`i''
		cap qui by RPS: replace b_product_`y`i'' = b_product_`y`i''[_n-1] if missing(b_product_`y`i'') & _n~= 1
	}	
	
	* keep if product exists in all 4 years, keep if city exist in all 4 years
	ds b_product* b_city*
	foreach var in `r(varlist)'{
		qui keep if `var' == 1
	}
	
	* generate D*xlp, where D is a dummy that's 1 if in the second period
	if "`dummy'"=="d_1997"{
		qui gen dummy = 1 if year2>= 1997
		qui replace dummy = 0 if dummy == .
	}
	if "`dummy'"=="METI_size"{
		qui gen dummy = METI_log_size
		qui replace dummy = 0 if dummy == .
	}
	if "`dummy'"=="METI_share"{
		qui gen dummy = METI_share
		qui replace dummy = 0 if dummy == .
	}
	
	if "`def'" ==  "x_rakuten" {
		gen xlp=xlp1_rakuten
		gen d_xlp1 = xlp*dummy
		local x_year "2010"
	}
	if "`def'" == "x_NS2004" {
		gen xlp=xlp1_NS2004
		gen d_xlp1 = xlp*dummy
		local x_year "2004"
	}
	if "`def'" == "x_NS2009" {
		gen xlp=xlp1_NS2009
		gen d_xlp1 = xlp*dummy
		local x_year "2009"
	}
	if "`def'" == "x_NS2014" {
		gen xlp=xlp1_NS2014
		gen d_xlp1 = xlp*dummy
		local x_year "2014"
	}
	
	gen xcatlp = xcatlp1_NS1999
	gen d_xcatlp1 = dummy*xcatlp1_NS1999
	gen d_p = dummy*avglp1
	
	* runs difference in difference specification
	if "`iv'" == "ols" {
		eststo: reghdfe delta_avgp avglp1 xlp d_xlp1 d_p, absorb(`fixed_effect', save) vce(cluster cityid_RPS RPS)
		estadd scalar r_squared `e(r2)'
	}
	if "`iv'" == "2sls" {
		eststo: ivreghdfe delta_avgp avglp1 d_p (xlp d_xlp1 = xcatlp d_xcatlp1), absorb(`fixed_effect') cluster(cityid_RPS RPS)  savefirst first saverf savefprefix(col3_st1)
		estadd scalar fs = e(widstat)
		estadd local iv "IV"
		estadd local pr "\{`y2',`y4'\}": col3_st1xlp
		estadd local pr "\{`y2',`y4'\}": col3_st1d_xlp1 
	}
	if "`iv'" == "2sls_2iv" {
		eststo: ivreghdfe delta_avgp avglp1 d_p (xlp d_xlp1 = xcatlp xraklp d_xcatlp1 d_xraklp1), absorb(`fixed_effect') cluster(cityid_RPS RPS)  savefirst first saverf savefprefix(col3_st1)
		estadd scalar fs = e(widstat)
		estadd local iv "IV (Cat, Rak)"
		estadd scalar jstat = e(j)
		estadd scalar jpval = e(jp)
		
		mat a = e(first)
		estadd scalar fst1 = a[4,1]: col3_st1xlp
		estadd scalar fst1 = a[4,2]: col3_st1d_xlp1 
		estadd scalar fs = e(widstat): col3_st1xlp
		estadd local iv "IV (Cat, Rak)": col3_st1xlp
		estadd scalar fs = e(widstat): col3_st1d_xlp1 
		estadd local iv "IV (Cat, Rak)": col3_st1d_xlp1 
	}
	
	* add statistics to output
	estadd local pr "\{`y2',`y4'\}"
	estadd local "x_t" `x_year'
	estadd local k `=`y2'-`y1''
	
end

*=================================================================================
* Diff-in-Diff annual regression specification
cap program drop annual_reg_PW
program define annual_reg_PW
args yr2_st yr2_end def fe goods iv dummy

	if "`fe'" == "city_product"{
		local fixed_effect = "citydum_t product_t"
	}
	if "`fe'" == "product"{
		local fixed_effect = "product_t"
	}
	
	qui use "$Data/pricereg//`=`yr2_st'-1'_`yr2_st'_PWreg.dta", clear
	forvalues y1 = `=`yr2_st''/`=`yr2_end'-1' {
		qui append using "$Data/pricereg//`y1'_`=`y1'+1'_PWreg.dta"
	}
	
	qui drop if TN_DW == 1 & "`goods'" == "tr"

	qui egen citydum_t = group(cityid_RPS year2)
	qui egen product_t = group(RPS year2)

	* limit to products and cities that exist in all relevant years
	forvalues year = `yr2_st'/`yr2_end' {
		cap qui gen b_city_`year' = 1 if (year2 == `year')& avglp1 ~= .
		sort cityid_RPS b_city_`year'
		cap qui by cityid_RPS: replace b_city_`year' = b_city_`year'[_n-1] if missing(b_city_`year') & _n~= 1
		
		cap qui gen b_product_`year' = 1 if (year2 == `year') & avglp1 ~= .
		sort RPS b_product_`year'
		cap qui by RPS: replace b_product_`year' = b_product_`year'[_n-1] if missing(b_product_`year') & _n~= 1
	}

	ds b_product* b_city*
	foreach var in `r(varlist)'{
		qui keep if `var' == 1
	}

	* generate dummy for period 2 interacte with xp
	if "`dummy'"=="d_1997"{
		qui gen dummy = 1 if year2>= 1997
		qui replace dummy = 0 if dummy == .
	}
	if "`dummy'"=="METI_size"{
		qui gen dummy = METI_log_size
		qui replace dummy = 0 if dummy == .
	}
	if "`dummy'"=="METI_share"{
		qui gen dummy = METI_share
		qui replace dummy = 0 if dummy == .
	}
	
	if "`def'" ==  "x_rakuten" {
		qui gen xlp=xlp1_rakuten
		qui gen d_xlp1 = xlp*dummy
		local x_year "2010"
	}
	if "`def'" == "x_NS2004" {
		qui gen xlp=xlp1_NS2004
		qui gen d_xlp1 = xlp*dummy
		local x_year "2004"
	}
	if "`def'" == "x_NS2009" {
		qui gen xlp=xlp1_NS2009
		qui gen d_xlp1 = xlp*dummy
		local x_year "2009"
	}
	if "`def'" == "x_NS2014" {
		qui gen xlp=xlp1_NS2014
		qui gen d_xlp1 = xlp*dummy
		local x_year "2014"
	}
	
	qui gen xcatlp = xcatlp1_NS1999
	qui gen d_xcatlp1 = dummy*xcatlp1_NS1999
	qui gen d_p = dummy*avglp1

	* runs difference in difference specification
	if "`iv'" == "ols" {
		eststo: reghdfe delta_avgp avglp1 xlp d_xlp1 d_p, absorb(`fixed_effect', save) vce(cluster cityid_RPS RPS) resid
		estadd local pr "Annual"
		estadd local "x_t" `x_year'
		estadd scalar r_squared `e(r2)'
	}
	if "`iv'" == "2sls" {
	
		eststo: ivreghdfe delta_avgp avglp1 d_p (xlp d_xlp1 = xcatlp d_xcatlp1), absorb(`fixed_effect', save resid(residuals)) cluster(cityid_RPS RPS) savefirst first saverf
		mat fstat = e(first)
		estadd local iv "IV"		
		estadd local pr "Annual"
		estadd local pr2 "`yr2_st'-`yr2_end'"
		estadd local "x_t" `x_year'
		estadd scalar fs = e(widstat)
		
		predict phat, xbd
		gen ehat = delta_avgp - phat
	
		rename (__hdfe1__ __hdfe2__) (beta_ct alpha_it)
		matrix est = e(b)
		gen gamma = est[1,3]
		gen delta1 = est[1,1]
		gen delta2 = est[1,4]
		gen delta3 = est[1,2]
	}
		
	if "`iv'" == "2sls_2iv" {
	
		eststo: ivreghdfe delta_avgp avglp1 d_p (xlp d_xlp1 = xcatlp d_xcatlp1 xraklp d_xraklp1), absorb(`fixed_effect', save resid(residuals)) cluster(cityid_RPS RPS) savefirst first saverf  savefprefix(col4_st1)
		mat fstat = e(first)
		estadd local iv "IV (Cat, Rak)"		
		estadd local pr "Annual"
		estadd local pr2 "`yr2_st'-`yr2_end'"
		estadd local "x_t" `x_year'
		estadd scalar fs = e(widstat)
		estadd scalar jstat = e(j)
		estadd scalar jpval = e(jp)
		
	
		
		mat a = e(first)
		estadd scalar fst1 = a[4,1]: col4_st1xlp
		estadd scalar fst1 = a[4,2]: col4_st1d_xlp1 
		estadd scalar fs = e(widstat): col4_st1xlp
		estadd local iv "IV (Cat, Rak)": col4_st1xlp
		estadd scalar fs = e(widstat): col4_st1d_xlp1 
		estadd local iv "IV (Cat, Rak)": col4_st1d_xlp1 
		estadd local pr "Annual": col4_st1xlp
		estadd local pr2 "`yr2_st'-`yr2_end'": col4_st1xlp
		estadd local pr "Annual": col4_st1d_xlp1 
		estadd local pr2 "`yr2_st'-`yr2_end'": col4_st1d_xlp1 
		

	}
		
end


*=================================================================================
* DinD specification: main results
cap program drop reg_dind_base_PW
program define reg_dind_base_PW
args def dummy no
	
	local fe "city_product"
	if "`fe'" == "city_product"{
		local fixed_effect = "citydum_t product_t"
	}
	if "`fe'" == "product"{
		local fixed_effect = "product_t"
	}
	if "`dummy'"=="d_1997" local suff2 ""
	if "`dummy'"=="METI_size" local suff2 "Msize_"
	
	
	foreach est in 2sls {
		est clear
		
		pricereg_PW 1991 1996 `def' "`fe'" tr `est' "`dummy'" /*column1 of table 5*/
		pricereg_PW 1996 2001 `def' "`fe'" tr `est' "`dummy'" /*column2 of table 5*/
		reg_dind_PW 1991 1996 1996 2001 `def' "`fe'" tr `est' "`dummy'" /*column3 of table 5*/
		annual_reg_PW 1992 2001 `def' "`fe'" tr `est' "`dummy'" /*column4 of table 5*/
		
		label var delta_avgp  "\$\Delta$ \$ (p_{ict}-p_{iTt})$"
		label var avglp1 "Lagged Price rel to Takamatsu"
		label var xlp "E-Commerce Intensity \$\times$ Lagged Price rel to Takamatsu"
		
		if "`dummy'"=="d_1997" {
			* label variables
			label var d_xlp1 "\$ D_{t}$ \$\times$ E-Commerce Intensity \$\times$ Lagged Price rel to Takamatsu"
			label var d_p "\$ D_{t}$ \$\times$ Lagged Price rel to Takamatsu"
		}
		if "`dummy'"=="METI_size" {
			* label variables
			label var d_xlp1 "EC Market Size \$\times$ E-Commerce Intensity \$\times$ Lagged Price rel to Takamatsu"
			label var d_p "EC Market Size \$\times$ Lagged Price rel to Takamatsu"
		}
				
		if "`fe'" == "city_product"{
			local suff = ""
		}
		if "`fe'" == "product"{
			local suff = "pd_"
		}
		
		if "`est'" == "ols"{
		esttab using "$table/table_`no'.tex", order(avglp1 xlp d_p d_xlp1) keep(avglp1 xlp d_xlp1 d_p) b(3) se(3) ar2(3) ///
				nolegend nonotes star(* 0.10 ** 0.05 *** 0.01) replace label wrap substitute(\_ _) ///
				stats(pr N r_squared, labels("\$t$" "Observations" "\$R^{2}$") fmt(%50 %9.0fc %9.2fc)) 
		}	
		if "`est'"=="2sls" {
		esttab using "$table/table_`no'.tex", order(avglp1 xlp d_p d_xlp1) keep(avglp1 xlp d_xlp1 d_p) b(3) se(3) ar2(3) ///
				nolegend nonotes star(* 0.10 ** 0.05 *** 0.01) replace label wrap substitute(\_ _) ///
				stats(pr N fs iv, labels("\$t$" "Observations" "First-stage F" "Estimation") fmt(%50 %9.0fc %9.2fc %50)) 
		}
				
		local ab filefilter "$table/table_`no'.tex" "$table/tmp.tex"
		local ba filefilter "$table/tmp.tex" "$table/table_`no'.tex"

		`ab', from ("Annual\BS\BS") to ("Annual\BS\BS\n&&&&1991-2001\BS\BS") replace
		`ba', from ("                    &\BSmulticolumn{1}{c}{$\BSDelta p_{ict}$}") to ("Dependent Variable&\BSmulticolumn{1}{c}{$\BSDelta p_{ict}$}") replace
		`ab', from ("\BSbegin{tabular}") to ("\BSadjustbox{max width=\BStextwidth}{\n\BSbegin{tabular}") replace
		`ba', from ("\BSend{tabular}") to ("\BSend{tabular}\n}") replace	
		`ab', from ("EC Market Size") to ("Log E-Commerce Market Size") replace		
		`ba', replace
		
		rm  "$table/tmp.tex"
	}
end
